


# @Time:2021/2/14 14:27
# @Author:andrew
# @email:zengjunjine1026@163.com
# @File:beike_redis.py
import time
import redis
from selenium import webdriver
from selenium.webdriver import ChromeOptions
import datetime
import pymongo
from utils.kafka_utils import producer


from settings import logger
from lxml import etree
from elasticsearch import Elasticsearch

# 默认host为localhost0,port为9200.但也可以指定host与port
es = Elasticsearch(hosts="localhost")
option = ChromeOptions()
option.add_argument('--disable-infobars')  # 禁用浏览器正在被自动化程序控制的提示
# 反爬机制代码开始，采用此代码在F12控制台输入window.navigator.webdriver结果不是True，而是undefined就成功了
option.add_experimental_option('excludeSwitches', ['enable-automation'])
option.add_argument('--no-sahttps://www.bilibili.com/video/BV1Cx411q7MM?from=search&seid=361786522620303182ndbox')
option.add_argument('--disable-dev-shm-usage')
# option.add_argument('--headless')
option.add_argument('blink-settings=imagesEnabled=false')
# option.add_argument('--disable-gpu')
driver = webdriver.Chrome(options=option)
driver.maximize_window()
import re

pool = redis.ConnectionPool(host='localhost', port=6379,db=2, decode_responses=True)
r = redis.Redis(connection_pool=pool)
find_float = lambda x: re.search("^([0-9]{1,})$", x).group()[0]


def parse_info(url, city_name):
    logger.info(url )
    driver.get(url + '/ershoufang' )
    with open('./stealth.min.js') as f:
        js = f.read()
        driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
            "source": js
        })
    time.sleep(0.1)
    driver.implicitly_wait(10)
    # logger.info(self.driver.save_screenshot('image.png'))
    # 往搜索框中输入iphone
    data = driver.page_source
    # logger.info(data)
    html = etree.HTML(data)


    for district in html.xpath("/html/body/div[1]/div[3]/div[1]/dl[2]/dd/div[1]/div[1]/a"):
        district_url = url + district.xpath("@href")[0]
        district_name= district.xpath("text()")[0]
        dic = {'district_url':district_url,"district_name":district_name,"city":city_name}
        r.sadd('all_district_set',str(dic))
        r.lpush('all_district_queue', str(dic))

if __name__ == "__main__":
    while True:

        url_and_city = r.rpoplpush("city_name_urls_queue2","city_name_urls_queue")
        url_and_city_dict = eval(url_and_city)
        url_and_district_dict = {'district_url': url_and_city_dict['city_url'] ,
                                 "city_name":url_and_city_dict['city_name'],
                                 }

        pages = parse_info(url_and_district_dict['district_url'],url_and_district_dict['city_name'],
                           )
